{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-11-05T11:27:12.792776Z",
     "start_time": "2019-11-05T11:27:11.890396Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "\n",
       "    <div class=\"bk-root\">\n",
       "        <a href=\"https://bokeh.pydata.org\" target=\"_blank\" class=\"bk-logo bk-logo-small bk-logo-notebook\"></a>\n",
       "        <span id=\"1001\">Loading BokehJS ...</span>\n",
       "    </div>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/javascript": [
       "\n",
       "(function(root) {\n",
       "  function now() {\n",
       "    return new Date();\n",
       "  }\n",
       "\n",
       "  var force = true;\n",
       "\n",
       "  if (typeof (root._bokeh_onload_callbacks) === \"undefined\" || force === true) {\n",
       "    root._bokeh_onload_callbacks = [];\n",
       "    root._bokeh_is_loading = undefined;\n",
       "  }\n",
       "\n",
       "  var JS_MIME_TYPE = 'application/javascript';\n",
       "  var HTML_MIME_TYPE = 'text/html';\n",
       "  var EXEC_MIME_TYPE = 'application/vnd.bokehjs_exec.v0+json';\n",
       "  var CLASS_NAME = 'output_bokeh rendered_html';\n",
       "\n",
       "  /**\n",
       "   * Render data to the DOM node\n",
       "   */\n",
       "  function render(props, node) {\n",
       "    var script = document.createElement(\"script\");\n",
       "    node.appendChild(script);\n",
       "  }\n",
       "\n",
       "  /**\n",
       "   * Handle when an output is cleared or removed\n",
       "   */\n",
       "  function handleClearOutput(event, handle) {\n",
       "    var cell = handle.cell;\n",
       "\n",
       "    var id = cell.output_area._bokeh_element_id;\n",
       "    var server_id = cell.output_area._bokeh_server_id;\n",
       "    // Clean up Bokeh references\n",
       "    if (id != null && id in Bokeh.index) {\n",
       "      Bokeh.index[id].model.document.clear();\n",
       "      delete Bokeh.index[id];\n",
       "    }\n",
       "\n",
       "    if (server_id !== undefined) {\n",
       "      // Clean up Bokeh references\n",
       "      var cmd = \"from bokeh.io.state import curstate; print(curstate().uuid_to_server['\" + server_id + \"'].get_sessions()[0].document.roots[0]._id)\";\n",
       "      cell.notebook.kernel.execute(cmd, {\n",
       "        iopub: {\n",
       "          output: function(msg) {\n",
       "            var id = msg.content.text.trim();\n",
       "            if (id in Bokeh.index) {\n",
       "              Bokeh.index[id].model.document.clear();\n",
       "              delete Bokeh.index[id];\n",
       "            }\n",
       "          }\n",
       "        }\n",
       "      });\n",
       "      // Destroy server and session\n",
       "      var cmd = \"import bokeh.io.notebook as ion; ion.destroy_server('\" + server_id + \"')\";\n",
       "      cell.notebook.kernel.execute(cmd);\n",
       "    }\n",
       "  }\n",
       "\n",
       "  /**\n",
       "   * Handle when a new output is added\n",
       "   */\n",
       "  function handleAddOutput(event, handle) {\n",
       "    var output_area = handle.output_area;\n",
       "    var output = handle.output;\n",
       "\n",
       "    // limit handleAddOutput to display_data with EXEC_MIME_TYPE content only\n",
       "    if ((output.output_type != \"display_data\") || (!output.data.hasOwnProperty(EXEC_MIME_TYPE))) {\n",
       "      return\n",
       "    }\n",
       "\n",
       "    var toinsert = output_area.element.find(\".\" + CLASS_NAME.split(' ')[0]);\n",
       "\n",
       "    if (output.metadata[EXEC_MIME_TYPE][\"id\"] !== undefined) {\n",
       "      toinsert[toinsert.length - 1].firstChild.textContent = output.data[JS_MIME_TYPE];\n",
       "      // store reference to embed id on output_area\n",
       "      output_area._bokeh_element_id = output.metadata[EXEC_MIME_TYPE][\"id\"];\n",
       "    }\n",
       "    if (output.metadata[EXEC_MIME_TYPE][\"server_id\"] !== undefined) {\n",
       "      var bk_div = document.createElement(\"div\");\n",
       "      bk_div.innerHTML = output.data[HTML_MIME_TYPE];\n",
       "      var script_attrs = bk_div.children[0].attributes;\n",
       "      for (var i = 0; i < script_attrs.length; i++) {\n",
       "        toinsert[toinsert.length - 1].firstChild.setAttribute(script_attrs[i].name, script_attrs[i].value);\n",
       "      }\n",
       "      // store reference to server id on output_area\n",
       "      output_area._bokeh_server_id = output.metadata[EXEC_MIME_TYPE][\"server_id\"];\n",
       "    }\n",
       "  }\n",
       "\n",
       "  function register_renderer(events, OutputArea) {\n",
       "\n",
       "    function append_mime(data, metadata, element) {\n",
       "      // create a DOM node to render to\n",
       "      var toinsert = this.create_output_subarea(\n",
       "        metadata,\n",
       "        CLASS_NAME,\n",
       "        EXEC_MIME_TYPE\n",
       "      );\n",
       "      this.keyboard_manager.register_events(toinsert);\n",
       "      // Render to node\n",
       "      var props = {data: data, metadata: metadata[EXEC_MIME_TYPE]};\n",
       "      render(props, toinsert[toinsert.length - 1]);\n",
       "      element.append(toinsert);\n",
       "      return toinsert\n",
       "    }\n",
       "\n",
       "    /* Handle when an output is cleared or removed */\n",
       "    events.on('clear_output.CodeCell', handleClearOutput);\n",
       "    events.on('delete.Cell', handleClearOutput);\n",
       "\n",
       "    /* Handle when a new output is added */\n",
       "    events.on('output_added.OutputArea', handleAddOutput);\n",
       "\n",
       "    /**\n",
       "     * Register the mime type and append_mime function with output_area\n",
       "     */\n",
       "    OutputArea.prototype.register_mime_type(EXEC_MIME_TYPE, append_mime, {\n",
       "      /* Is output safe? */\n",
       "      safe: true,\n",
       "      /* Index of renderer in `output_area.display_order` */\n",
       "      index: 0\n",
       "    });\n",
       "  }\n",
       "\n",
       "  // register the mime type if in Jupyter Notebook environment and previously unregistered\n",
       "  if (root.Jupyter !== undefined) {\n",
       "    var events = require('base/js/events');\n",
       "    var OutputArea = require('notebook/js/outputarea').OutputArea;\n",
       "\n",
       "    if (OutputArea.prototype.mime_types().indexOf(EXEC_MIME_TYPE) == -1) {\n",
       "      register_renderer(events, OutputArea);\n",
       "    }\n",
       "  }\n",
       "\n",
       "  \n",
       "  if (typeof (root._bokeh_timeout) === \"undefined\" || force === true) {\n",
       "    root._bokeh_timeout = Date.now() + 5000;\n",
       "    root._bokeh_failed_load = false;\n",
       "  }\n",
       "\n",
       "  var NB_LOAD_WARNING = {'data': {'text/html':\n",
       "     \"<div style='background-color: #fdd'>\\n\"+\n",
       "     \"<p>\\n\"+\n",
       "     \"BokehJS does not appear to have successfully loaded. If loading BokehJS from CDN, this \\n\"+\n",
       "     \"may be due to a slow or bad network connection. Possible fixes:\\n\"+\n",
       "     \"</p>\\n\"+\n",
       "     \"<ul>\\n\"+\n",
       "     \"<li>re-rerun `output_notebook()` to attempt to load from CDN again, or</li>\\n\"+\n",
       "     \"<li>use INLINE resources instead, as so:</li>\\n\"+\n",
       "     \"</ul>\\n\"+\n",
       "     \"<code>\\n\"+\n",
       "     \"from bokeh.resources import INLINE\\n\"+\n",
       "     \"output_notebook(resources=INLINE)\\n\"+\n",
       "     \"</code>\\n\"+\n",
       "     \"</div>\"}};\n",
       "\n",
       "  function display_loaded() {\n",
       "    var el = document.getElementById(\"1001\");\n",
       "    if (el != null) {\n",
       "      el.textContent = \"BokehJS is loading...\";\n",
       "    }\n",
       "    if (root.Bokeh !== undefined) {\n",
       "      if (el != null) {\n",
       "        el.textContent = \"BokehJS \" + root.Bokeh.version + \" successfully loaded.\";\n",
       "      }\n",
       "    } else if (Date.now() < root._bokeh_timeout) {\n",
       "      setTimeout(display_loaded, 100)\n",
       "    }\n",
       "  }\n",
       "\n",
       "\n",
       "  function run_callbacks() {\n",
       "    try {\n",
       "      root._bokeh_onload_callbacks.forEach(function(callback) { callback() });\n",
       "    }\n",
       "    finally {\n",
       "      delete root._bokeh_onload_callbacks\n",
       "    }\n",
       "    console.info(\"Bokeh: all callbacks have finished\");\n",
       "  }\n",
       "\n",
       "  function load_libs(js_urls, callback) {\n",
       "    root._bokeh_onload_callbacks.push(callback);\n",
       "    if (root._bokeh_is_loading > 0) {\n",
       "      console.log(\"Bokeh: BokehJS is being loaded, scheduling callback at\", now());\n",
       "      return null;\n",
       "    }\n",
       "    if (js_urls == null || js_urls.length === 0) {\n",
       "      run_callbacks();\n",
       "      return null;\n",
       "    }\n",
       "    console.log(\"Bokeh: BokehJS not loaded, scheduling load and callback at\", now());\n",
       "    root._bokeh_is_loading = js_urls.length;\n",
       "    for (var i = 0; i < js_urls.length; i++) {\n",
       "      var url = js_urls[i];\n",
       "      var s = document.createElement('script');\n",
       "      s.src = url;\n",
       "      s.async = false;\n",
       "      s.onreadystatechange = s.onload = function() {\n",
       "        root._bokeh_is_loading--;\n",
       "        if (root._bokeh_is_loading === 0) {\n",
       "          console.log(\"Bokeh: all BokehJS libraries loaded\");\n",
       "          run_callbacks()\n",
       "        }\n",
       "      };\n",
       "      s.onerror = function() {\n",
       "        console.warn(\"failed to load library \" + url);\n",
       "      };\n",
       "      console.log(\"Bokeh: injecting script tag for BokehJS library: \", url);\n",
       "      document.getElementsByTagName(\"head\")[0].appendChild(s);\n",
       "    }\n",
       "  };var element = document.getElementById(\"1001\");\n",
       "  if (element == null) {\n",
       "    console.log(\"Bokeh: ERROR: autoload.js configured with elementid '1001' but no matching script tag was found. \")\n",
       "    return false;\n",
       "  }\n",
       "\n",
       "  var js_urls = [\"https://cdn.pydata.org/bokeh/release/bokeh-1.0.0.min.js\", \"https://cdn.pydata.org/bokeh/release/bokeh-widgets-1.0.0.min.js\", \"https://cdn.pydata.org/bokeh/release/bokeh-tables-1.0.0.min.js\", \"https://cdn.pydata.org/bokeh/release/bokeh-gl-1.0.0.min.js\"];\n",
       "\n",
       "  var inline_js = [\n",
       "    function(Bokeh) {\n",
       "      Bokeh.set_log_level(\"info\");\n",
       "    },\n",
       "    \n",
       "    function(Bokeh) {\n",
       "      \n",
       "    },\n",
       "    function(Bokeh) {\n",
       "      console.log(\"Bokeh: injecting CSS: https://cdn.pydata.org/bokeh/release/bokeh-1.0.0.min.css\");\n",
       "      Bokeh.embed.inject_css(\"https://cdn.pydata.org/bokeh/release/bokeh-1.0.0.min.css\");\n",
       "      console.log(\"Bokeh: injecting CSS: https://cdn.pydata.org/bokeh/release/bokeh-widgets-1.0.0.min.css\");\n",
       "      Bokeh.embed.inject_css(\"https://cdn.pydata.org/bokeh/release/bokeh-widgets-1.0.0.min.css\");\n",
       "      console.log(\"Bokeh: injecting CSS: https://cdn.pydata.org/bokeh/release/bokeh-tables-1.0.0.min.css\");\n",
       "      Bokeh.embed.inject_css(\"https://cdn.pydata.org/bokeh/release/bokeh-tables-1.0.0.min.css\");\n",
       "    }\n",
       "  ];\n",
       "\n",
       "  function run_inline_js() {\n",
       "    \n",
       "    if ((root.Bokeh !== undefined) || (force === true)) {\n",
       "      for (var i = 0; i < inline_js.length; i++) {\n",
       "        inline_js[i].call(root, root.Bokeh);\n",
       "      }if (force === true) {\n",
       "        display_loaded();\n",
       "      }} else if (Date.now() < root._bokeh_timeout) {\n",
       "      setTimeout(run_inline_js, 100);\n",
       "    } else if (!root._bokeh_failed_load) {\n",
       "      console.log(\"Bokeh: BokehJS failed to load within specified timeout.\");\n",
       "      root._bokeh_failed_load = true;\n",
       "    } else if (force !== true) {\n",
       "      var cell = $(document.getElementById(\"1001\")).parents('.cell').data().cell;\n",
       "      cell.output_area.append_execute_result(NB_LOAD_WARNING)\n",
       "    }\n",
       "\n",
       "  }\n",
       "\n",
       "  if (root._bokeh_is_loading === 0) {\n",
       "    console.log(\"Bokeh: BokehJS loaded, going straight to plotting\");\n",
       "    run_inline_js();\n",
       "  } else {\n",
       "    load_libs(js_urls, function() {\n",
       "      console.log(\"Bokeh: BokehJS plotting callback run at\", now());\n",
       "      run_inline_js();\n",
       "    });\n",
       "  }\n",
       "}(window));"
      ],
      "application/vnd.bokehjs_load.v0+json": "\n(function(root) {\n  function now() {\n    return new Date();\n  }\n\n  var force = true;\n\n  if (typeof (root._bokeh_onload_callbacks) === \"undefined\" || force === true) {\n    root._bokeh_onload_callbacks = [];\n    root._bokeh_is_loading = undefined;\n  }\n\n  \n\n  \n  if (typeof (root._bokeh_timeout) === \"undefined\" || force === true) {\n    root._bokeh_timeout = Date.now() + 5000;\n    root._bokeh_failed_load = false;\n  }\n\n  var NB_LOAD_WARNING = {'data': {'text/html':\n     \"<div style='background-color: #fdd'>\\n\"+\n     \"<p>\\n\"+\n     \"BokehJS does not appear to have successfully loaded. If loading BokehJS from CDN, this \\n\"+\n     \"may be due to a slow or bad network connection. Possible fixes:\\n\"+\n     \"</p>\\n\"+\n     \"<ul>\\n\"+\n     \"<li>re-rerun `output_notebook()` to attempt to load from CDN again, or</li>\\n\"+\n     \"<li>use INLINE resources instead, as so:</li>\\n\"+\n     \"</ul>\\n\"+\n     \"<code>\\n\"+\n     \"from bokeh.resources import INLINE\\n\"+\n     \"output_notebook(resources=INLINE)\\n\"+\n     \"</code>\\n\"+\n     \"</div>\"}};\n\n  function display_loaded() {\n    var el = document.getElementById(\"1001\");\n    if (el != null) {\n      el.textContent = \"BokehJS is loading...\";\n    }\n    if (root.Bokeh !== undefined) {\n      if (el != null) {\n        el.textContent = \"BokehJS \" + root.Bokeh.version + \" successfully loaded.\";\n      }\n    } else if (Date.now() < root._bokeh_timeout) {\n      setTimeout(display_loaded, 100)\n    }\n  }\n\n\n  function run_callbacks() {\n    try {\n      root._bokeh_onload_callbacks.forEach(function(callback) { callback() });\n    }\n    finally {\n      delete root._bokeh_onload_callbacks\n    }\n    console.info(\"Bokeh: all callbacks have finished\");\n  }\n\n  function load_libs(js_urls, callback) {\n    root._bokeh_onload_callbacks.push(callback);\n    if (root._bokeh_is_loading > 0) {\n      console.log(\"Bokeh: BokehJS is being loaded, scheduling callback at\", now());\n      return null;\n    }\n    if (js_urls == null || js_urls.length === 0) {\n      run_callbacks();\n      return null;\n    }\n    console.log(\"Bokeh: BokehJS not loaded, scheduling load and callback at\", now());\n    root._bokeh_is_loading = js_urls.length;\n    for (var i = 0; i < js_urls.length; i++) {\n      var url = js_urls[i];\n      var s = document.createElement('script');\n      s.src = url;\n      s.async = false;\n      s.onreadystatechange = s.onload = function() {\n        root._bokeh_is_loading--;\n        if (root._bokeh_is_loading === 0) {\n          console.log(\"Bokeh: all BokehJS libraries loaded\");\n          run_callbacks()\n        }\n      };\n      s.onerror = function() {\n        console.warn(\"failed to load library \" + url);\n      };\n      console.log(\"Bokeh: injecting script tag for BokehJS library: \", url);\n      document.getElementsByTagName(\"head\")[0].appendChild(s);\n    }\n  };var element = document.getElementById(\"1001\");\n  if (element == null) {\n    console.log(\"Bokeh: ERROR: autoload.js configured with elementid '1001' but no matching script tag was found. \")\n    return false;\n  }\n\n  var js_urls = [\"https://cdn.pydata.org/bokeh/release/bokeh-1.0.0.min.js\", \"https://cdn.pydata.org/bokeh/release/bokeh-widgets-1.0.0.min.js\", \"https://cdn.pydata.org/bokeh/release/bokeh-tables-1.0.0.min.js\", \"https://cdn.pydata.org/bokeh/release/bokeh-gl-1.0.0.min.js\"];\n\n  var inline_js = [\n    function(Bokeh) {\n      Bokeh.set_log_level(\"info\");\n    },\n    \n    function(Bokeh) {\n      \n    },\n    function(Bokeh) {\n      console.log(\"Bokeh: injecting CSS: https://cdn.pydata.org/bokeh/release/bokeh-1.0.0.min.css\");\n      Bokeh.embed.inject_css(\"https://cdn.pydata.org/bokeh/release/bokeh-1.0.0.min.css\");\n      console.log(\"Bokeh: injecting CSS: https://cdn.pydata.org/bokeh/release/bokeh-widgets-1.0.0.min.css\");\n      Bokeh.embed.inject_css(\"https://cdn.pydata.org/bokeh/release/bokeh-widgets-1.0.0.min.css\");\n      console.log(\"Bokeh: injecting CSS: https://cdn.pydata.org/bokeh/release/bokeh-tables-1.0.0.min.css\");\n      Bokeh.embed.inject_css(\"https://cdn.pydata.org/bokeh/release/bokeh-tables-1.0.0.min.css\");\n    }\n  ];\n\n  function run_inline_js() {\n    \n    if ((root.Bokeh !== undefined) || (force === true)) {\n      for (var i = 0; i < inline_js.length; i++) {\n        inline_js[i].call(root, root.Bokeh);\n      }if (force === true) {\n        display_loaded();\n      }} else if (Date.now() < root._bokeh_timeout) {\n      setTimeout(run_inline_js, 100);\n    } else if (!root._bokeh_failed_load) {\n      console.log(\"Bokeh: BokehJS failed to load within specified timeout.\");\n      root._bokeh_failed_load = true;\n    } else if (force !== true) {\n      var cell = $(document.getElementById(\"1001\")).parents('.cell').data().cell;\n      cell.output_area.append_execute_result(NB_LOAD_WARNING)\n    }\n\n  }\n\n  if (root._bokeh_is_loading === 0) {\n    console.log(\"Bokeh: BokehJS loaded, going straight to plotting\");\n    run_inline_js();\n  } else {\n    load_libs(js_urls, function() {\n      console.log(\"Bokeh: BokehJS plotting callback run at\", now());\n      run_inline_js();\n    });\n  }\n}(window));"
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import torch\n",
    "import torch.nn as nn\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import torchvision.datasets as dsets\n",
    "import torchvision.transforms as transforms\n",
    "from torch.utils.data import DataLoader\n",
    "\n",
    "from bokeh.io import show, output_notebook\n",
    "from bokeh.plotting import figure, gridplot\n",
    "from bokeh.models import LinearAxis, Range1d\n",
    "output_notebook()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### GPU\n",
    "指定使用的 GPU 编号。  \n",
    "`watch -n 1 nvidia-smi` 实时查看 GPU 的运行状态。 "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-11-05T11:27:13.059238Z",
     "start_time": "2019-11-05T11:27:12.795024Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "3"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "torch.cuda.set_device(\"cuda:3\")\n",
    "torch.cuda.current_device()\n",
    "# device = torch.device(\"cuda:5\")\n",
    "# xxx.to(device)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-11-02T15:34:18.036850Z",
     "start_time": "2019-11-02T15:34:18.032937Z"
    }
   },
   "source": [
    "### Data\n",
    "通过`torchvision.datasets`下载`CIFAR-10`数据。其中包含10类图片，每类图片6000张。  \n",
    "训练集：`train=True`  \n",
    "测试集：`train=False`  \n",
    "常用的还有`torchvision.datasets.ImageFolder()`，按文件夹取图片。  \n",
    "\n",
    "`torchvision.transforms`可以对图片做处理。  \n",
    "`trans` 对四种处理进行了组合，包括调节图片尺寸、随机水平翻转、随机中心切割、化为`Tensor`."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-11-05T11:27:13.162781Z",
     "start_time": "2019-11-05T11:27:13.061526Z"
    }
   },
   "outputs": [],
   "source": [
    "trans = transforms.Compose([\n",
    "    transforms.Resize(40),\n",
    "    transforms.RandomHorizontalFlip(),\n",
    "    transforms.RandomCrop(32),\n",
    "    transforms.ToTensor(),\n",
    "])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-11-05T11:27:15.142944Z",
     "start_time": "2019-11-05T11:27:13.164804Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Files already downloaded and verified\n",
      "Files already downloaded and verified\n"
     ]
    }
   ],
   "source": [
    "train_dataset = dsets.CIFAR10(root='../dataset', train=True, transform=trans, download=True)\n",
    "test_dataset = dsets.CIFAR10(root='../dataset', train=False, transform=trans, download=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-11-05T11:27:15.172770Z",
     "start_time": "2019-11-05T11:27:15.163135Z"
    }
   },
   "outputs": [],
   "source": [
    "batch_size = 100\n",
    "train_loader = DataLoader(dataset=train_dataset, batch_size=batch_size, shuffle=True)\n",
    "test_loader = DataLoader(dataset=test_dataset, batch_size=batch_size, shuffle=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-11-05T11:27:15.431197Z",
     "start_time": "2019-11-05T11:27:15.174839Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.image.AxesImage at 0x7ff229c7cfd0>"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAP8AAAD8CAYAAAC4nHJkAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvOIA7rQAAH3FJREFUeJztnVuMXNd1pv9Vt67qezf7QrJJiRJ1GcmxRMmMIEiZjB3PBIoRRDaQZOwHQw9GGAQxEAPJg+AAYw8wD/ZgbMMPAw/okRJl4PFlfImFQJjEEWwIiQNFlCXrHomiKLHJVrPJ7mZ3dVXXdc1DlyZUa/+bJTZZTWn/H0B0ca/a56zaddY5VeevtZa5O4QQ6ZHZbgeEENuDgl+IRFHwC5EoCn4hEkXBL0SiKPiFSBQFvxCJouAXIlEU/EIkSm4rk83sHgBfB5AF8D/d/Uux5+fzee8rFoO2VqtF52UQ/hVi1vi+Cjl+XstHbLlsltrMwjs0i5xDIz42m/w1x353mY35SH6x2fY231eb780ykRcQod0Ov7aY79HtRfy3yCIzWybiRzbD3092DABAO/JrWY8dCGxOdHthFpdXUa6sd7Wziw5+M8sC+O8A/gOAWQBPmNnD7v4Cm9NXLOLA7R8K2paXF+m++jLhN368wBfnqh391DY5PkBtE6OD1FbI5oPjub4SnYMsX+LFpWVqqzf5axsbHaG2TKsRHK/VanTO+vo6tRVL4ZM1ALTAT16Vajk4PjI6TOfA+fbqtTq1ZRF+XwB+shka5O/zwAA/PvJ5vh7ViI8eu0BkwsdI7DU3PRzfX37gB3w/m3fb9TPfyR0Ajrr7MXevA/gOgHu3sD0hRA/ZSvDPADhx3v9nO2NCiPcAW/nOH/rc8Y7PqmZ2CMAhAOjr69vC7oQQl5KtXPlnAew97/97AJza/CR3P+zuB939YC7Pv5sJIXrLVoL/CQDXm9k1ZlYA8EkAD18at4QQl5uL/tjv7k0z+yyAv8WG1Peguz8fm7O+vo7nXwg/ZfnMGTpvnNxgtR38zutEa4jarDRFbWttrjqUW+E78G4FOqeyzu/YVqr8DnyjxaWtMxGNs5gL+9hs8u1lyd1mIP5VrbK+Rm3Ndvh12/oOOicTUQEbEbWilOPHQZncMV9sNemc/n5+t98y/NOrETUIABCRDyvrYYWm2QiPA0A2F35fGutV7sMmtqTzu/sjAB7ZyjaEENuDfuEnRKIo+IVIFAW/EImi4BciURT8QiTKlu72v1syAEo5IlNFfvx3NZH09k3zBJepyXFqK8WknEjWVrUWToBZb3AZyiPbK5QiCUGRxB5v8/2NjIcTmpoNvr1CnvsRSbZEtsDftFo9vFaNJl+P/sj2cgPcx2JkXtPCcmQmkiXYjGTgxTJJBwd4Mll5rUJtjWZY0oslVK6unAuOt2Nv2Obtd/1MIcT7CgW/EImi4BciURT8QiSKgl+IROnp3X4zR9HCCRVDQ9yVG2bGguM7SjwTJN/mpanKizzZptXm58NqJex7huf1YDhSFiwXuUu9fG6Vz4u8a+ND4TvOqys8CaceSdCpkqQTIF6XbpCUwmrUeeJJpsVfWD6SYNQipcsAIEduz9dqfE4hz9/QTJsnBNXKS9QGkhQGAH3kMG62uSJxbi2s+LQi9Rg3oyu/EImi4BciURT8QiSKgl+IRFHwC5EoCn4hEqWnUl/ODGN94V2WIlLOCEnqmBzmNdNapF0UgEifGSCbixSSI3XYau2I1BTR5XKR5JJWjUtinuXn7NOnw12AWg3+qlcrPOmk0uKy6GAp0n2nRtp1gb/mjHGZKtsX6ZSzxmXd/nzYx1ykFdZ6pO5itcGlvnakydpymfu4XAkfP2UiLQPAeiN8DNQjtRo3oyu/EImi4BciURT8QiSKgl+IRFHwC5EoCn4hEmVLUp+ZHQewig31rOnuB6M7yxomR8OSzVCeS2zFYtiWyXJppRSpj9doctmrHclUcw9LQPVIvb1WncuAbY9kzEUkNs/xrLPVejhDr9Xi61uJtAZrRmyra9z/k4thP/IZvr3hMl/7xpu8nVv1HJcqr5q4Ljg+NbWHzrGhcH08AKgtnaW2cplnR55b5VLfmXNhWff4Ce5HKxsO3Vqdy4ObuRQ6/0fcnb8zQogrEn3sFyJRthr8DuDvzOxJMzt0KRwSQvSGrX7sv9vdT5nZFICfmNlL7v7Y+U/onBQOAUAx8r1eCNFbtnTld/dTnb+nAfwIwB2B5xx294PufrCQ07cMIa4ULjoazWzAzIbeegzgNwE8d6kcE0JcXrbysX8awI867a1yAP63u//f2IR8Lovdk+HCjsMFLlEM9oelLYtIZYhkWFkkm65W5bJRhsiAO4Z427CBAZ6NtnKOiyQjwzxjbjVSVPP1k+Ftlmv8K1chkgg20x/JSszzzMPjZ8PZhTWPFF2NZPWNDA9R2103c4V5ZS4s63olsq8Jni1aq/D1KJf5tbQvz7e5d2f4tU1NTdM58yth6fDsy2/SOZu56OB392MAbr3Y+UKI7UVfwoVIFAW/EImi4BciURT8QiSKgl+IROltAc+sYXwonG2Xq4elIQDoy4fd7O8L96UDgFqVy2GNSL+10dFwX0AAcFL0sd7i59BGI1JccpD38Tu1EO7FBgCvvs6zvRZWw68tUgsSV0d6Hn783x6gtj27uP/ff/JYcPyfjnIpqtnmmYy5DJfmVpcXqK1SDq/j0BCX3tDi2YXFIp9XINmnANBvfF6zFX5zrtq7m84ZWgz3cnzmNb4Wm9GVX4hEUfALkSgKfiESRcEvRKIo+IVIlN7e7c/lMDW+I2irLvK74hkLu1kmbY4AoBqpZZazSD27SFsrdqasNvhd6tExnqBTb/E72MdmT1Hb4gr3kdX3y0ZafA0X+famcuG7ygBQXOSKxPXDO4Pjc+Pcj/nl09RWq/A1furll6ktQ9pXNQYircZGeEINMjxkRka4+jTUjrQHI3Uevb5C5+wjCXJ9+e6v57ryC5EoCn4hEkXBL0SiKPiFSBQFvxCJouAXIlF6LPXlMTYxGbSNDfL2WplMOClieWWJzmmslfn2WrF2XbygnZMEo8FBXqevAW578RiXqNZqvPVTsdjHbYWwj6UBLkONZbks+uTReWpr1vnhUxsJS32TY3w9DFx+azS5FFyp81qCa6RWX73JX7NFpNtINzfkM5FWb5lI7cJceB2bNS6lOpGJSe5ZEF35hUgUBb8QiaLgFyJRFPxCJIqCX4hEUfALkSgXlPrM7EEAvw3gtLv/SmdsHMB3AewDcBzA77s7193+dWsAke0s0s6I0Repp9aPcNYTAOQi57xMJlKPj8iAfSXeruvMmzwrrnKGL9m141wSq3HVC0Ui6d24f4bOyUQ22MzyNV6JSK25bLjO4FCBvy87xvZT2/7rr6K21954gtpeevlkcLyQi8hozmXiZpOHTIZkVAJAvsDXsd0OH1ftiK5oFj5OI0rkO+jmyv+XAO7ZNHY/gEfd/XoAj3b+L4R4D3HB4Hf3xwAsbhq+F8BDnccPAfj4JfZLCHGZudjv/NPuPgcAnb9Tl84lIUQvuOw3/MzskJkdMbMjq5XIl1UhRE+52OCfN7NdAND5S+svufthdz/o7geH+vlNLCFEb7nY4H8YwH2dx/cB+PGlcUcI0Su6kfq+DeDDACbMbBbAFwB8CcD3zOwzAN4A8Hvd7Kztjup6uFihNXhmFhDOwFpb4wUO6w1+Xmtm+CeQcoVLcyvENrOXL6M3+faunuDCzP7dXBqqrPN5MzfcGhwvOP/KtXSOF0ItjYYLrgIAzvJMtb07dwXHl9d4tuK1/+Z6ahse41mJw2M3UdvSQnj9l87xlmf5iByZcZ5R2WhHskV5sihajfDxHUkSpK3j3kVS34WD390/RUwffRf7EUJcYegXfkIkioJfiERR8AuRKAp+IRJFwS9EovS0gKfD0bKwHOItXlCRyRqlIi/6OTjEpaFTC1xWfG12gdpy+bAfhXneV299nm/v+iku5330w1z2evXk5lSLf2VoJlwgdWJHuKAmAJxe4EU6R0cjsleb+18gBStPL4Sz7AAgV1ymtoXlOWo7Ocez8PL58HEwOsy1t2qVC2ae49dLi2hz7YgMmLHwPItkmEbaPHaNrvxCJIqCX4hEUfALkSgKfiESRcEvRKIo+IVIlJ5KfdlsBqOjg0FbM8elvnI5nJHmDS6fnFvlWVuvv8GlrXKZy0alYvhcOfcazy6cLvKijjMzV1Pb6O5rqC2/GkkRI0VN99x6B5/yJpffSk0uVbbAMwXX1sK2Xf1hKRIA6i3+umwgfNwAwJ6B3dQ2NBqWOFfPvknnnJ4/S20N4/Lmep0XBUWGa3MDfeEs03o1ImGSgqBGZMOgS10/UwjxvkLBL0SiKPiFSBQFvxCJouAXIlF6ere/3WpidTl8JzVX57Xu8qQ1EXgJOeSy3FgpcyVgbIgnsowOhO/KVpf43f6p3bwG3swt/47anputU9vLR7ntrl3jwfHlZT5nen+47h8AZFChtnqNKwGjHr5zv3Ka30kv1XktwV3j4dcFAMstXlcvf8tYcLwaSRT6x0ceprbZE/w1ZyMtuWKNtFgeUSPWVq4RXiuWBBfcRtfPFEK8r1DwC5EoCn4hEkXBL0SiKPiFSBQFvxCJ0k27rgcB/DaA0+7+K52xLwL4AwBv6R6fd/dHutlhligerUgSgxOZJEPaeAFAy7jUt8QVJaysROq31cJy2a4RLg/+6kc+Qm17bryT2n74Fw9S285Ikku2Hq5PePLYq3x7195MbcUd11HbgHN5trIY7t1aaoelNwCoV7mseGaV20YneRLUjp37guPV8jCdk+EmtAo8mSlWw6/R4FKrNcMJauY8ca3ZDIfupZb6/hLAPYHxr7n7gc6/rgJfCHHlcMHgd/fHAPBysUKI9yRb+c7/WTN7xsweNDP+WU4IcUVyscH/DQD7ARwAMAfgK+yJZnbIzI6Y2ZFyhX/vEUL0losKfnefd/eWu7cBfBMALRPj7ofd/aC7Hxzs51VthBC95aKC38x2nfffTwB47tK4I4ToFd1Ifd8G8GEAE2Y2C+ALAD5sZgcAOIDjAP6wm50ZACNKRItkKQG8bVGkcxK8GtlepATe+A7e5mtnf1havP3gDXTOTXdxOW/pNJc3+5o88/DaPXuorU1e3M4pXjuvuc4l00okG7De5PMa1fCh1QKXKV89OUttzz53hNruupP7uGNnOKtyZTUsRQIA6fAFAJjYx2Xddqy9Vj0i2xEJ+dwCb19WWw072SbZlCEuGPzu/qnA8ANd70EIcUWiX/gJkSgKfiESRcEvRKIo+IVIFAW/EInS0wKe7kCbZDBVa1yiKJAstlyOF0zMZrj8c91O/mvkYomfD/ddvTc4fuuv8cy9XTfeQm1P/9NfUNtVe7mPOz/wQWorTO4Pjuf6R+icyjqXHKsrPHNv/tQJaluaD8t2rQbPzisNhQukAsDEBH+vT5x6itqmd80Ex5uVSBZplbfdsrUlamt5OKMSAJxp3ABKfeHXVtjJX/NKH8l0fRcRrSu/EImi4BciURT8QiSKgl+IRFHwC5EoCn4hEqWnUp+ZIZ8N73IpUqCxtR6WNUr9JTonm+HSylQkc+/EHM+k2n97qJQhsOeD4fENuGTXWF2jtpEhLs1N3nCA2tZy4Z52zz/1BJ1Tq3I/Vlb4epw5+Qa1ZVthqbVY5IfczDVhWQ4AbrmBFxJtZnmmXT47Gh4v8KzP3Dov0ll5/SS1MRkbAJqRy2yZ9JXs38Ff1zTpAZnPd38915VfiERR8AuRKAp+IRJFwS9Eoij4hUiU3ib2tNuoVcN3Uvv7uCtWDN8NzWd4DTlvcVtpkLfy+p3/+DvUdtdvfTQ4PjwxTefMH3uR2rIR/5dXeQ2/heP/Qm2nVsN3nH/2139N5wyWeALJeo0nwOyc5orE8FD4TvVrszwZqB5Zj/Hd+6jthg9+iNrQ6gsOLy7zeoEVoi4BwFKV+2jOj+H1Kk9cK5MWW17mqsNNYRED7e67denKL0SqKPiFSBQFvxCJouAXIlEU/EIkioJfiETppl3XXgB/BWAngDaAw+7+dTMbB/BdAPuw0bLr992dFzgD4HC0ndTWa/OkCGuGZZKmR1pyRWqmFfuGqe3Ah7hs1JcPS2IvPM1ryC2depXaajUu5awuLVLbiaMvUFvZw8lO+Rbf12COS5/DRZ5cMjnGpb65+TeD481IW7bKKpcVT7zGk4iA56mlXA7XICzm+PHR7JuitrNNfuyUSrwGYf8QT0Ir5cJy5Gplhc5ptsOS47tQ+rq68jcB/Km73wTgTgB/bGY3A7gfwKPufj2ARzv/F0K8R7hg8Lv7nLv/ovN4FcCLAGYA3Avgoc7THgLw8cvlpBDi0vOuvvOb2T4AtwF4HMC0u88BGycIAPyzkhDiiqPr4DezQQA/APA5d+dfRt4575CZHTGzI2tVXktfCNFbugp+M8tjI/C/5e4/7AzPm9mujn0XgGDDc3c/7O4H3f3gQKlwKXwWQlwCLhj8ZmYAHgDwort/9TzTwwDu6zy+D8CPL717QojLRTdZfXcD+DSAZ83s6c7Y5wF8CcD3zOwzAN4A8HsX3pRjQy18J+0m/0qQy4dr7rUiNdPq4NlX0yO8rt7fPvw31DY+HZaUpnaF23gBQL3Cs/Py+bDEAwCDA1xSymW4NDdA5MidU+GabwBQXeUKbSnLfTy7cIbaGvXwezNU5JJXvcylvleeOkJtcy+9TG21Jmmhledr2Iqt7x4ufWKAH8OZPi61FolsNwa+Vjd94JrgeKl4jM7ZzAWD393/AQDLcQznuAohrnj0Cz8hEkXBL0SiKPiFSBQFvxCJouAXIlF6WsATbmi3w8JBIZJZVsyR4ocZXmjRIy2c2nWeWXbmTDgbDQDKC2FbqcF/8NgGf13jY1x+G909SW3NVo3aTp4K++iRfK9Mhh8G9SaXTLPGC38OFMPyLEnQ3NhezBjJ0mzVuZyaIcfbSoXLm/U+Ig8CGNrN136txFubrba5DLi+Fr4G7xi+ls6ZINJtLt99SOvKL0SiKPiFSBQFvxCJouAXIlEU/EIkioJfiETprdQHQ8bCWWLFPp7B5CRDb6AUlpMAYGBogtoqDZ5htWOI1xzIET/q5+bpnHaGb6+S59LW9HQ4awsA2nUuG914y57g+M9/+iidU/cKteWNy6nVMp83PBTOSizk+CGXtUg/u3X+nr02x2W75eXwe1azNTpn8gZ+TZwZjWQlOn+vl87wtSqshyXTgZlIJmYlnDXZjqilm9GVX4hEUfALkSgKfiESRcEvRKIo+IVIlJ7e7c8YUMiFzzeVGk+YyJKWUe1IfblKgydnZPM8SaSvwO/m5vNhPwr9vG3VyDBPMHpzgasElZnwXXsAmNp7HbWdPB2uq/eBX72bzikvnKK2Yy/zVlhrZZ7IksuG139khNcmNFLfEQDmTnIf33g9ktjTF17/4WmuFE2OR3yMqA62yN/rsSUeajNT48HxPaP8GDj6QjiBq1blSWub0ZVfiERR8AuRKAp+IRJFwS9Eoij4hUgUBb8QiXJBqc/M9gL4KwA7sdFr67C7f93MvgjgDwAsdJ76eXd/JLqznGF6Mny+aZw9S+dVW2EJaI3nZsAzvJVXLpJcMjzMkykKpBVWdY3X8CvFaqrVue3Iz39ObdfeyCXC2dmwBJSJ1Dvs7+O1+LIRObVU4tLWWjks9VWrXIJtRlq2DZa4H3fddgO1FUmCUTPLaxO2GjwJp3qCS32Z1SK1TfUPUdttN3wgPGd0ms55cu614HizwV/XZrrR+ZsA/tTdf2FmQwCeNLOfdGxfc/f/1vXehBBXDN306psDMNd5vGpmLwKYudyOCSEuL+/qO7+Z7QNwG4DHO0OfNbNnzOxBM+Otb4UQVxxdB7+ZDQL4AYDPufsKgG8A2A/gADY+GXyFzDtkZkfM7MhKhX+nE0L0lq6C38zy2Aj8b7n7DwHA3efdveXubQDfBHBHaK67H3b3g+5+cLifVzoRQvSWCwa/mRmABwC86O5fPW9813lP+wSA5y69e0KIy0U3d/vvBvBpAM+a2dOdsc8D+JSZHQDgAI4D+MMLbahQMFy1N3z1HzEukxw9EZZe5hd4dl69xaWhwUH+stcqPEOs1S4Hx7ORc+jiApcwV8tclllvcD+yzm1Dg+FbL/NvLtI5s2tcvmo7lwinJ7ksau1wdtnSMq+31zfA37PRES6VFbJ8/Wt1IvnmuLy5VuPbq5cjLcrafN51e3dS2+6d4XU8Mcsl3bML4ZhoxlqebaKbu/3/ACB0BEQ1fSHElY1+4SdEoij4hUgUBb8QiaLgFyJRFPxCJEpPC3hmc4bhMZIZR6QLABibyoYNA7wI45l5XhB0PdLuKlfgxRvZtHaDZxA2WtyPc1Uuew1EstjWK1yaq66HC3jWIz62IjZ3svYAyiuRdl3D4UKow8O82Gm1yrd35ixfq8FBnl1omfD1zZpcJi7keBHXPq5Io1Dga7Xvun3UVq2EfXnssRfonGdePh3e1nr3WX268guRKAp+IRJFwS9Eoij4hUgUBb8QiaLgFyJReir1mRlyxfAui8M81398MHyOylW5jJYv8eymlUjfNLT4+bBUnApPyfN9tWq8n12hn/uRz/H1yGa5xFnzsC/1Bpc3PZK5Z1wRg9e55Ngipnwkmw4FLm8uL3Gpr1rn/elGRsPSbY5IgACQiax9BVxKmz+zSm1LkQzO1bVwlubf/+wlvi+iiq7XJfUJIS6Agl+IRFHwC5EoCn4hEkXBL0SiKPiFSJSeSn3ttqHMCiBmB+m8wYGwbpQvcR1qIJJ+NTLCpbnyCu8lV14JF1QsVyJZfevcNlTgBTCLpC8gADRrXOLM5cLn80LkNJ/v49loZnxif6QQaoaYmi0uRRVKkR6Ko1zeXFzkEtsqkT6Hx/naVyI9A185zguyvvTsCWqbHufZotN7yGvL8ON0ghQ0nV/lsuc7Nt/1M4UQ7ysU/EIkioJfiERR8AuRKAp+IRLlgnf7zawI4DEAfZ3nf9/dv2Bm1wD4DoBxAL8A8Gl3j7bhrdeB2dfDttoyvzs/NBm+Q1wsRRI6uHiA8XH+sstrvI7c8nLYtnSWJ4Is8ZvDyLb5Xfa2cyWj1eIKAtphW+wsbxme2JPN8bWqRpKgnNzUz5M2XgDQrPCWYq1Ifb9WJFlouRyex7p4AcBiRPE5fpS/octn16itvsZ3uHMk3Mrrpqtn6Bzm4itvrtA5m+nmyl8D8Bvufis22nHfY2Z3AvgygK+5+/UAlgB8puu9CiG2nQsGv2/wVofKfOefA/gNAN/vjD8E4OOXxUMhxGWhq+/8ZpbtdOg9DeAnAF4FsOz+/z/czQLgn1GEEFccXQW/u7fc/QCAPQDuAHBT6GmhuWZ2yMyOmNmRc2Ve/EEI0Vve1d1+d18G8DMAdwIYNbO37gbtAXCKzDns7gfd/eDIYKTjgRCip1ww+M1s0sxGO49LAP49gBcB/BTA73aedh+AH18uJ4UQl55uEnt2AXjIzLLYOFl8z93/xsxeAPAdM/svAJ4C8MCFNuSWQys/EbQ1CgfpvFo7nMiSaYZbUwFAcYTLV6OT/BPIWIYnnoxXwokWy4u8vdPyGS7nVdf48reaXD6E83N2uxn2cb3Kv3IVCpF6gTnu/+o6Tzypkq94+YgaPJQJJ6sAQDvDJaxGg69j30BYMi3meb3A0QL38VqMUtsHb+Vtw2685VZq23fddcHxO+7k8ubsqXJw/B9f5TGxmQsGv7s/A+C2wPgxbHz/F0K8B9Ev/IRIFAW/EImi4BciURT8QiSKgl+IRDGPZI9d8p2ZLQB4K69vAkD3usTlQ368Hfnxdt5rflzt7pPdbLCnwf+2HZsdcXcu7ssP+SE/Lqsf+tgvRKIo+IVIlO0M/sPbuO/zkR9vR368nfetH9v2nV8Isb3oY78QibItwW9m95jZv5jZUTO7fzt86Phx3MyeNbOnzexID/f7oJmdNrPnzhsbN7OfmNkrnb9j2+THF83sZGdNnjazj/XAj71m9lMze9HMnjezP+mM93RNIn70dE3MrGhm/2xmv+z48Z8749eY2eOd9fiumUVSP7vA3Xv6D0AWG2XArgVQAPBLADf32o+OL8cBTGzDfn8dwO0Anjtv7L8CuL/z+H4AX94mP74I4M96vB67ANzeeTwE4GUAN/d6TSJ+9HRNABiAwc7jPIDHsVFA53sAPtkZ/x8A/mgr+9mOK/8dAI66+zHfKPX9HQD3boMf24a7PwZgc53qe7FRCBXoUUFU4kfPcfc5d/9F5/EqNorFzKDHaxLxo6f4Bpe9aO52BP8MgPPbmW5n8U8H8Hdm9qSZHdomH95i2t3ngI2DEMDUNvryWTN7pvO14LJ//TgfM9uHjfoRj2Mb12STH0CP16QXRXO3I/hDJXa2S3K4291vB/BbAP7YzH59m/y4kvgGgP3Y6NEwB+ArvdqxmQ0C+AGAz7l7990nLr8fPV8T30LR3G7ZjuCfBbD3vP/T4p+XG3c/1fl7GsCPsL2ViebNbBcAdP6e3g4n3H2+c+C1AXwTPVoTM8tjI+C+5e4/7Az3fE1CfmzXmnT2/a6L5nbLdgT/EwCu79y5LAD4JICHe+2EmQ2Y2dBbjwH8JoDn4rMuKw9joxAqsI0FUd8Ktg6fQA/WxMwMGzUgX3T3r55n6umaMD96vSY9K5rbqzuYm+5mfgwbd1JfBfDn2+TDtdhQGn4J4Ple+gHg29j4+NjAxiehzwDYAeBRAK90/o5vkx//C8CzAJ7BRvDt6oEfv4aNj7DPAHi68+9jvV6TiB89XRMAt2CjKO4z2DjR/Kfzjtl/BnAUwP8B0LeV/egXfkIkin7hJ0SiKPiFSBQFvxCJouAXIlEU/EIkioJfiERR8AuRKAp+IRLl/wHCOW2RBgdIrQAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.imshow(train_dataset.data[0])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Model\n",
    "构造 Block，并用它作为单元构造 ResNet."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-11-05T11:27:15.437039Z",
     "start_time": "2019-11-05T11:27:15.433196Z"
    }
   },
   "outputs": [],
   "source": [
    "def conv3x3(in_channels, out_channels, stride=1):\n",
    "    return nn.Conv2d(in_channels, out_channels, kernel_size=3, stride=stride, padding=1, bias=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Residual 体现在最后 `return` 时，返回的是经过卷积操作的 `x` 加上原来的 `x`。  \n",
    "但由于两者维度可能不相符(卷积的 `stride > 1` 会使得图像降维)，对 `x` 要进行 `downsample` 操作。  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-11-05T11:27:15.536134Z",
     "start_time": "2019-11-05T11:27:15.440304Z"
    }
   },
   "outputs": [],
   "source": [
    "class ResidualBlock(nn.Module):\n",
    "    def __init__(self, in_channels, out_channels, stride=1, downsample=None):\n",
    "        super().__init__()\n",
    "        self.downsample = downsample\n",
    "        self.conv1 = conv3x3(in_channels, out_channels, stride)\n",
    "        self.bn1 = nn.BatchNorm2d(out_channels)\n",
    "        self.relu = nn.ReLU(inplace=True)\n",
    "        self.conv2 = conv3x3(out_channels, out_channels, stride)\n",
    "        self.bn2 = nn.BatchNorm2d(out_channels)\n",
    "    def forward(self, x):\n",
    "        residual = self.downsample(x) if self.downsample else x\n",
    "        x = self.relu(self.bn1(self.conv1(x)))\n",
    "        x = self.bn2(self.conv2(x))\n",
    "        return self.relu(x + residual)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-11-05T11:27:15.677658Z",
     "start_time": "2019-11-05T11:27:15.538373Z"
    }
   },
   "outputs": [],
   "source": [
    "class ResNet(nn.Module):\n",
    "    def __init__(self, block, layers, num_classes=10):\n",
    "        super().__init__()\n",
    "        self.in_channels = 16\n",
    "        self.hidden_channels = 16\n",
    "        self.conv = conv3x3(3, 16)\n",
    "        self.bn = nn.BatchNorm2d(16)\n",
    "        self.relu = nn.ReLU(inplace=True)\n",
    "        self.layer1 = self.make_layer(block, 16, layers[0])\n",
    "        self.layer2 = self.make_layer(block, 32, layers[1])\n",
    "        self.layer3 = self.make_layer(block, 64, layers[2], 2)\n",
    "        self.avg_pool = nn.AvgPool2d(8)\n",
    "        self.fc = nn.Linear(64, num_classes)\n",
    "    def forward(self, x):\n",
    "        x = self.relu(self.bn(self.conv(x)))\n",
    "        x = self.layer1(x)\n",
    "        x = self.layer2(x)\n",
    "        x = self.layer3(x)\n",
    "        x = self.avg_pool(x)\n",
    "        x = torch.squeeze(x)\n",
    "        return self.fc(x)\n",
    "    def make_layer(self, block, out_channels, blocks_size, stride=1):\n",
    "        downsample = None\n",
    "        if stride != 1:\n",
    "            downsample = nn.Sequential(\n",
    "                conv3x3(self.hidden_channels, out_channels, stride * 2),\n",
    "                nn.BatchNorm2d(out_channels),\n",
    "            )\n",
    "        elif self.hidden_channels != out_channels:\n",
    "            downsample = nn.Sequential(\n",
    "                conv3x3(self.hidden_channels, out_channels, stride),\n",
    "                nn.BatchNorm2d(out_channels),\n",
    "            )\n",
    "        layers = []\n",
    "        layers.append(block(self.hidden_channels, out_channels, stride, downsample))\n",
    "        self.hidden_channels = out_channels\n",
    "        for i in range(1, blocks_size):\n",
    "            layers.append(block(out_channels, out_channels))\n",
    "        return nn.Sequential(*layers)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-11-05T11:27:22.194387Z",
     "start_time": "2019-11-05T11:27:15.680496Z"
    }
   },
   "outputs": [],
   "source": [
    "lrate = 0.001\n",
    "epochs = 3\n",
    "\n",
    "model = ResNet(ResidualBlock, [4,4,4]).cuda()\n",
    "criterion = nn.CrossEntropyLoss()\n",
    "optim = torch.optim.Adam(model.parameters(), lr=lrate)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "网络层数越深，可覆盖的解空间越广，理论上的精度越高。但实际上层数适中时才能保证收敛性和精度。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-11-05T11:27:22.202903Z",
     "start_time": "2019-11-05T11:27:22.197221Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "ResNet(\n",
       "  (conv): Conv2d(3, 16, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "  (bn): BatchNorm2d(16, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "  (relu): ReLU(inplace=True)\n",
       "  (layer1): Sequential(\n",
       "    (0): ResidualBlock(\n",
       "      (conv1): Conv2d(16, 16, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "      (bn1): BatchNorm2d(16, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (relu): ReLU(inplace=True)\n",
       "      (conv2): Conv2d(16, 16, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "      (bn2): BatchNorm2d(16, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "    )\n",
       "    (1): ResidualBlock(\n",
       "      (conv1): Conv2d(16, 16, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "      (bn1): BatchNorm2d(16, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (relu): ReLU(inplace=True)\n",
       "      (conv2): Conv2d(16, 16, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "      (bn2): BatchNorm2d(16, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "    )\n",
       "    (2): ResidualBlock(\n",
       "      (conv1): Conv2d(16, 16, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "      (bn1): BatchNorm2d(16, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (relu): ReLU(inplace=True)\n",
       "      (conv2): Conv2d(16, 16, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "      (bn2): BatchNorm2d(16, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "    )\n",
       "    (3): ResidualBlock(\n",
       "      (conv1): Conv2d(16, 16, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "      (bn1): BatchNorm2d(16, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (relu): ReLU(inplace=True)\n",
       "      (conv2): Conv2d(16, 16, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "      (bn2): BatchNorm2d(16, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "    )\n",
       "  )\n",
       "  (layer2): Sequential(\n",
       "    (0): ResidualBlock(\n",
       "      (downsample): Sequential(\n",
       "        (0): Conv2d(16, 32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "        (1): BatchNorm2d(32, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      )\n",
       "      (conv1): Conv2d(16, 32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "      (bn1): BatchNorm2d(32, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (relu): ReLU(inplace=True)\n",
       "      (conv2): Conv2d(32, 32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "      (bn2): BatchNorm2d(32, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "    )\n",
       "    (1): ResidualBlock(\n",
       "      (conv1): Conv2d(32, 32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "      (bn1): BatchNorm2d(32, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (relu): ReLU(inplace=True)\n",
       "      (conv2): Conv2d(32, 32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "      (bn2): BatchNorm2d(32, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "    )\n",
       "    (2): ResidualBlock(\n",
       "      (conv1): Conv2d(32, 32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "      (bn1): BatchNorm2d(32, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (relu): ReLU(inplace=True)\n",
       "      (conv2): Conv2d(32, 32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "      (bn2): BatchNorm2d(32, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "    )\n",
       "    (3): ResidualBlock(\n",
       "      (conv1): Conv2d(32, 32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "      (bn1): BatchNorm2d(32, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (relu): ReLU(inplace=True)\n",
       "      (conv2): Conv2d(32, 32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "      (bn2): BatchNorm2d(32, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "    )\n",
       "  )\n",
       "  (layer3): Sequential(\n",
       "    (0): ResidualBlock(\n",
       "      (downsample): Sequential(\n",
       "        (0): Conv2d(32, 64, kernel_size=(3, 3), stride=(4, 4), padding=(1, 1), bias=False)\n",
       "        (1): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      )\n",
       "      (conv1): Conv2d(32, 64, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), bias=False)\n",
       "      (bn1): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (relu): ReLU(inplace=True)\n",
       "      (conv2): Conv2d(64, 64, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), bias=False)\n",
       "      (bn2): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "    )\n",
       "    (1): ResidualBlock(\n",
       "      (conv1): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "      (bn1): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (relu): ReLU(inplace=True)\n",
       "      (conv2): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "      (bn2): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "    )\n",
       "    (2): ResidualBlock(\n",
       "      (conv1): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "      (bn1): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (relu): ReLU(inplace=True)\n",
       "      (conv2): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "      (bn2): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "    )\n",
       "    (3): ResidualBlock(\n",
       "      (conv1): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "      (bn1): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (relu): ReLU(inplace=True)\n",
       "      (conv2): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "      (bn2): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "    )\n",
       "  )\n",
       "  (avg_pool): AvgPool2d(kernel_size=8, stride=8, padding=0)\n",
       "  (fc): Linear(in_features=64, out_features=10, bias=True)\n",
       ")"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "注意：每次反向传播的时候都需要将参数的梯度归零。  \n",
    "`optim.step()`则在每个`Variable`的`grad`都被计算出来后，更新每个`Variable`的数值"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "在每次训练中都用`train_loader`中的一个`batch`作为训练数据。  \n",
    "`Tensor.cuda()` 每个 `batch` 在实际使用之前，都先移入 `GPU` 后进行计算。  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-11-05T11:28:43.859301Z",
     "start_time": "2019-11-05T11:27:22.204809Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "\n",
       "\n",
       "\n",
       "\n",
       "\n",
       "\n",
       "  <div class=\"bk-root\" id=\"0fe7fd5e-63e0-45d8-960c-fec9ed5b8fec\"></div>\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/javascript": [
       "(function(root) {\n",
       "  function embed_document(root) {\n",
       "    \n",
       "  var docs_json = {\"fb4b199b-5535-4f87-a7f3-71e53ec7da25\":{\"roots\":{\"references\":[{\"attributes\":{\"below\":[{\"id\":\"1011\",\"type\":\"LinearAxis\"}],\"left\":[{\"id\":\"1016\",\"type\":\"LinearAxis\"}],\"renderers\":[{\"id\":\"1011\",\"type\":\"LinearAxis\"},{\"id\":\"1015\",\"type\":\"Grid\"},{\"id\":\"1016\",\"type\":\"LinearAxis\"},{\"id\":\"1020\",\"type\":\"Grid\"},{\"id\":\"1029\",\"type\":\"BoxAnnotation\"},{\"id\":\"1039\",\"type\":\"GlyphRenderer\"}],\"title\":{\"id\":\"1042\",\"type\":\"Title\"},\"toolbar\":{\"id\":\"1027\",\"type\":\"Toolbar\"},\"x_range\":{\"id\":\"1003\",\"type\":\"DataRange1d\"},\"x_scale\":{\"id\":\"1007\",\"type\":\"LinearScale\"},\"y_range\":{\"id\":\"1005\",\"type\":\"DataRange1d\"},\"y_scale\":{\"id\":\"1009\",\"type\":\"LinearScale\"}},\"id\":\"1002\",\"subtype\":\"Figure\",\"type\":\"Plot\"},{\"attributes\":{\"dimension\":1,\"plot\":{\"id\":\"1002\",\"subtype\":\"Figure\",\"type\":\"Plot\"},\"ticker\":{\"id\":\"1017\",\"type\":\"BasicTicker\"}},\"id\":\"1020\",\"type\":\"Grid\"},{\"attributes\":{\"source\":{\"id\":\"1036\",\"type\":\"ColumnDataSource\"}},\"id\":\"1040\",\"type\":\"CDSView\"},{\"attributes\":{\"line_alpha\":0.1,\"line_color\":\"#1f77b4\",\"x\":{\"field\":\"x\"},\"y\":{\"field\":\"y\"}},\"id\":\"1038\",\"type\":\"Line\"},{\"attributes\":{},\"id\":\"1021\",\"type\":\"PanTool\"},{\"attributes\":{},\"id\":\"1022\",\"type\":\"WheelZoomTool\"},{\"attributes\":{\"callback\":null},\"id\":\"1003\",\"type\":\"DataRange1d\"},{\"attributes\":{\"overlay\":{\"id\":\"1029\",\"type\":\"BoxAnnotation\"}},\"id\":\"1023\",\"type\":\"BoxZoomTool\"},{\"attributes\":{\"callback\":null,\"data\":{\"x\":[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29],\"y\":[2.5870320796966553,1.6724653244018555,1.7372021675109863,1.417607307434082,1.5630276203155518,1.3479342460632324,1.4241915941238403,1.273922085762024,1.262518048286438,1.3388592004776,1.3069524765014648,1.0766123533248901,0.9742487072944641,1.1288620233535767,0.9749751091003418,1.1721603870391846,0.9923550486564636,0.955566942691803,1.1654319763183594,0.8107395768165588,0.7889630198478699,0.7996874451637268,0.9163007140159607,0.9184238314628601,0.9086265563964844,0.8164898157119751,0.9570517539978027,0.809384822845459,1.0100897550582886,0.8508542776107788]},\"selected\":{\"id\":\"1048\",\"type\":\"Selection\"},\"selection_policy\":{\"id\":\"1047\",\"type\":\"UnionRenderers\"}},\"id\":\"1036\",\"type\":\"ColumnDataSource\"},{\"attributes\":{},\"id\":\"1024\",\"type\":\"SaveTool\"},{\"attributes\":{},\"id\":\"1012\",\"type\":\"BasicTicker\"},{\"attributes\":{},\"id\":\"1025\",\"type\":\"ResetTool\"},{\"attributes\":{},\"id\":\"1026\",\"type\":\"HelpTool\"},{\"attributes\":{\"formatter\":{\"id\":\"1043\",\"type\":\"BasicTickFormatter\"},\"plot\":{\"id\":\"1002\",\"subtype\":\"Figure\",\"type\":\"Plot\"},\"ticker\":{\"id\":\"1012\",\"type\":\"BasicTicker\"}},\"id\":\"1011\",\"type\":\"LinearAxis\"},{\"attributes\":{\"active_drag\":\"auto\",\"active_inspect\":\"auto\",\"active_multi\":null,\"active_scroll\":\"auto\",\"active_tap\":\"auto\",\"tools\":[{\"id\":\"1021\",\"type\":\"PanTool\"},{\"id\":\"1022\",\"type\":\"WheelZoomTool\"},{\"id\":\"1023\",\"type\":\"BoxZoomTool\"},{\"id\":\"1024\",\"type\":\"SaveTool\"},{\"id\":\"1025\",\"type\":\"ResetTool\"},{\"id\":\"1026\",\"type\":\"HelpTool\"}]},\"id\":\"1027\",\"type\":\"Toolbar\"},{\"attributes\":{\"data_source\":{\"id\":\"1036\",\"type\":\"ColumnDataSource\"},\"glyph\":{\"id\":\"1037\",\"type\":\"Line\"},\"hover_glyph\":null,\"muted_glyph\":null,\"nonselection_glyph\":{\"id\":\"1038\",\"type\":\"Line\"},\"selection_glyph\":null,\"view\":{\"id\":\"1040\",\"type\":\"CDSView\"}},\"id\":\"1039\",\"type\":\"GlyphRenderer\"},{\"attributes\":{\"callback\":null},\"id\":\"1005\",\"type\":\"DataRange1d\"},{\"attributes\":{\"bottom_units\":\"screen\",\"fill_alpha\":{\"value\":0.5},\"fill_color\":{\"value\":\"lightgrey\"},\"left_units\":\"screen\",\"level\":\"overlay\",\"line_alpha\":{\"value\":1.0},\"line_color\":{\"value\":\"black\"},\"line_dash\":[4,4],\"line_width\":{\"value\":2},\"plot\":null,\"render_mode\":\"css\",\"right_units\":\"screen\",\"top_units\":\"screen\"},\"id\":\"1029\",\"type\":\"BoxAnnotation\"},{\"attributes\":{\"line_color\":\"#1f77b4\",\"x\":{\"field\":\"x\"},\"y\":{\"field\":\"y\"}},\"id\":\"1037\",\"type\":\"Line\"},{\"attributes\":{\"plot\":null,\"text\":\"\"},\"id\":\"1042\",\"type\":\"Title\"},{\"attributes\":{},\"id\":\"1007\",\"type\":\"LinearScale\"},{\"attributes\":{},\"id\":\"1009\",\"type\":\"LinearScale\"},{\"attributes\":{},\"id\":\"1043\",\"type\":\"BasicTickFormatter\"},{\"attributes\":{},\"id\":\"1045\",\"type\":\"BasicTickFormatter\"},{\"attributes\":{\"plot\":{\"id\":\"1002\",\"subtype\":\"Figure\",\"type\":\"Plot\"},\"ticker\":{\"id\":\"1012\",\"type\":\"BasicTicker\"}},\"id\":\"1015\",\"type\":\"Grid\"},{\"attributes\":{},\"id\":\"1047\",\"type\":\"UnionRenderers\"},{\"attributes\":{\"formatter\":{\"id\":\"1045\",\"type\":\"BasicTickFormatter\"},\"plot\":{\"id\":\"1002\",\"subtype\":\"Figure\",\"type\":\"Plot\"},\"ticker\":{\"id\":\"1017\",\"type\":\"BasicTicker\"}},\"id\":\"1016\",\"type\":\"LinearAxis\"},{\"attributes\":{},\"id\":\"1048\",\"type\":\"Selection\"},{\"attributes\":{},\"id\":\"1017\",\"type\":\"BasicTicker\"}],\"root_ids\":[\"1002\"]},\"title\":\"Bokeh Application\",\"version\":\"1.0.0\"}};\n",
       "  var render_items = [{\"docid\":\"fb4b199b-5535-4f87-a7f3-71e53ec7da25\",\"roots\":{\"1002\":\"0fe7fd5e-63e0-45d8-960c-fec9ed5b8fec\"}}];\n",
       "  root.Bokeh.embed.embed_items_notebook(docs_json, render_items);\n",
       "\n",
       "  }\n",
       "  if (root.Bokeh !== undefined) {\n",
       "    embed_document(root);\n",
       "  } else {\n",
       "    var attempts = 0;\n",
       "    var timer = setInterval(function(root) {\n",
       "      if (root.Bokeh !== undefined) {\n",
       "        embed_document(root);\n",
       "        clearInterval(timer);\n",
       "      }\n",
       "      attempts++;\n",
       "      if (attempts > 100) {\n",
       "        console.log(\"Bokeh: ERROR: Unable to run BokehJS code because BokehJS library is missing\");\n",
       "        clearInterval(timer);\n",
       "      }\n",
       "    }, 10, root)\n",
       "  }\n",
       "})(window);"
      ],
      "application/vnd.bokehjs_exec.v0+json": ""
     },
     "metadata": {
      "application/vnd.bokehjs_exec.v0+json": {
       "id": "1002"
      }
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "result = []\n",
    "for e in range(epochs):\n",
    "    for i, (inputs, targets) in enumerate(train_loader):\n",
    "        inputs = inputs.cuda()\n",
    "        targets = targets.cuda()\n",
    "        optim.zero_grad()\n",
    "        outputs = model(inputs)\n",
    "        loss = criterion(outputs, targets)\n",
    "        loss.backward()\n",
    "        optim.step()\n",
    "        if i%50==0:\n",
    "            result.append(float(loss))\n",
    "    \n",
    "fig = figure()\n",
    "fig.line(range(len(result)), result)\n",
    "show(fig)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Result"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "`re = torch.max(Tensor,dim)`, 返回的re为一个二维向量，其中`re[0]`为最大值的`Tensor`，re[1]为最大值对应的`index`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-11-05T11:28:46.717338Z",
     "start_time": "2019-11-05T11:28:43.861064Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy of the model on the 10000 test images: 70.85 %\n"
     ]
    }
   ],
   "source": [
    "correct = 0\n",
    "total = 0\n",
    "for i, (inputs, targets) in enumerate(test_loader):\n",
    "    targets = targets.cuda()\n",
    "    inputs = inputs.cuda()\n",
    "    outputs = model(inputs)\n",
    "    _, preds = torch.max(outputs.data, 1)\n",
    "    total += len(outputs)\n",
    "    correct += (preds == targets).sum()\n",
    "accuracy = 100 * correct.double() / total\n",
    "print('Accuracy of the model on the 10000 test images: %.2f %%' % (accuracy))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-11-03T07:12:44.915877Z",
     "start_time": "2019-11-03T07:12:44.913744Z"
    }
   },
   "source": [
    "### Save Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-11-05T11:28:46.781174Z",
     "start_time": "2019-11-05T11:28:46.718966Z"
    }
   },
   "outputs": [],
   "source": [
    "torch.save(model.state_dict(), 'resnet_cuda.pkl')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.1"
  },
  "varInspector": {
   "cols": {
    "lenName": 16,
    "lenType": 16,
    "lenVar": 40
   },
   "kernels_config": {
    "python": {
     "delete_cmd_postfix": "",
     "delete_cmd_prefix": "del ",
     "library": "var_list.py",
     "varRefreshCmd": "print(var_dic_list())"
    },
    "r": {
     "delete_cmd_postfix": ") ",
     "delete_cmd_prefix": "rm(",
     "library": "var_list.r",
     "varRefreshCmd": "cat(var_dic_list()) "
    }
   },
   "oldHeight": 511.99678000000006,
   "position": {
    "height": "533.984px",
    "left": "411.875px",
    "right": "20px",
    "top": "161.984px",
    "width": "760.516px"
   },
   "types_to_exclude": [
    "module",
    "function",
    "builtin_function_or_method",
    "instance",
    "_Feature"
   ],
   "varInspector_section_display": "block",
   "window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
